

<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" />
  <meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />

  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  
  <title>Block Devices and Nomad &mdash; Ceph Documentation</title>
  

  
  <link rel="stylesheet" href="../../_static/ceph.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/ceph.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/graphviz.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/custom.css" type="text/css" />

  
  

  
  

  

  
  <!--[if lt IE 9]>
    <script src="../../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
        <script src="../../_static/jquery.js"></script>
        <script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
        <script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
        <script src="../../_static/doctools.js"></script>
        <script src="../../_static/sphinx_highlight.js"></script>
    
    <script type="text/javascript" src="../../_static/js/theme.js"></script>

    
    <link rel="index" title="Index" href="../../genindex/" />
    <link rel="search" title="Search" href="../../search/" />
    <link rel="next" title="块设备与 OpenStack" href="../rbd-openstack/" />
    <link rel="prev" title="Block Devices and Kubernetes" href="../rbd-kubernetes/" /> 
</head>

<body class="wy-body-for-nav">

   
  <header class="top-bar">
    <div role="navigation" aria-label="Page navigation">
  <ul class="wy-breadcrumbs">
      <li><a href="../../" class="icon icon-home" aria-label="Home"></a></li>
          <li class="breadcrumb-item"><a href="../">Ceph 块设备</a></li>
          <li class="breadcrumb-item"><a href="../rbd-integrations/">Ceph 块设备与第三方对接</a></li>
      <li class="breadcrumb-item active">Block Devices and Nomad</li>
      <li class="wy-breadcrumbs-aside">
            <a href="../../_sources/rbd/rbd-nomad.rst.txt" rel="nofollow"> View page source</a>
      </li>
  </ul>
  <hr/>
</div>
  </header>
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search"  style="background: #eee" >
          

          
            <a href="../../" class="icon icon-home"> Ceph
          

          
          </a>

          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../search/" method="get">
    <input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../start/">Ceph 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../install/">安装 Ceph</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cephadm/">Cephadm</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../rados/">Ceph 存储集群</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cephfs/">Ceph 文件系统</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../">Ceph 块设备</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../rados-rbd-cmds/">基本命令</a></li>
<li class="toctree-l2"><a class="reference internal" href="../rbd-operations/">运维</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="../rbd-integrations/">对接</a><ul class="current">
<li class="toctree-l3"><a class="reference internal" href="../rbd-ko/">内核模块</a></li>
<li class="toctree-l3"><a class="reference internal" href="../qemu-rbd/">QEMU</a></li>
<li class="toctree-l3"><a class="reference internal" href="../libvirt/">libvirt</a></li>
<li class="toctree-l3"><a class="reference internal" href="../rbd-kubernetes/">Kubernetes</a></li>
<li class="toctree-l3 current"><a class="current reference internal" href="#">Nomad</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#create-a-pool">Create a Pool</a></li>
<li class="toctree-l4"><a class="reference internal" href="#configure-ceph-csi">Configure ceph-csi</a></li>
<li class="toctree-l4"><a class="reference internal" href="#create-ceph-csi-controller-and-plugin-nodes">Create ceph-csi controller and plugin nodes</a></li>
<li class="toctree-l4"><a class="reference internal" href="#using-ceph-block-devices">Using Ceph Block Devices</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="../rbd-openstack/">OpenStack</a></li>
<li class="toctree-l3"><a class="reference internal" href="../rbd-cloudstack/">CloudStack</a></li>
<li class="toctree-l3"><a class="reference internal" href="../iscsi-overview/">LIO iSCSI Gateway</a></li>
<li class="toctree-l3"><a class="reference internal" href="../rbd-windows/">Windows</a></li>
<li class="toctree-l3"><a class="reference internal" href="../nvmeof-overview/">NVMe-oF 网关</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../man/">手册页</a></li>
<li class="toctree-l2"><a class="reference internal" href="../api/">APIs</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../radosgw/">Ceph 对象网关</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../mgr/">Ceph 管理器守护进程</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../mgr/dashboard/">Ceph 仪表盘</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../monitoring/">监控概览</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../api/">API 文档</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../architecture/">体系结构</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/developer_guide/">开发者指南</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/internals/">Ceph 内幕</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../governance/">项目管理</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../foundation/">Ceph 基金会</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../ceph-volume/">ceph-volume</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../releases/general/">Ceph 版本（总目录）</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../releases/">Ceph 版本（索引）</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../security/">Security</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../hardware-monitoring/">硬件监控</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../glossary/">Ceph 术语</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../jaegertracing/">Tracing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../translation_cn/">中文版翻译资源</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../">Ceph</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
<div id="dev-warning" class="admonition note">
  <p class="first admonition-title">Notice</p>
  <p class="last">This document is for a development version of Ceph.</p>
</div>
  <div id="docubetter" align="right" style="padding: 5px; font-weight: bold;">
    <a href="https://pad.ceph.com/p/Report_Documentation_Bugs">Report a Documentation Bug</a>
  </div>

  
  <section id="block-devices-and-nomad">
<h1>Block Devices and Nomad<a class="headerlink" href="#block-devices-and-nomad" title="Permalink to this heading"></a></h1>
<p>Like Kubernetes, Nomad can use Ceph Block Device. This is made possible by
<a class="reference external" href="https://github.com/ceph/ceph-csi/">ceph-csi</a>, which allows you to dynamically provision RBD images or import
existing RBD images.</p>
<p>Every version of Nomad is compatible with <a class="reference external" href="https://github.com/ceph/ceph-csi/">ceph-csi</a>, but the reference
version of Nomad that was used to generate the procedures and guidance in this
document is Nomad v1.1.2, the latest version available at the time of the
writing of the document.</p>
<p>To use Ceph Block Devices with Nomad, you must install
and configure <code class="docutils literal notranslate"><span class="pre">ceph-csi</span></code> within your Nomad environment. The following
diagram shows the Nomad/Ceph technology stack.</p>
<p class="ditaa">
<img src="../../_images/ditaa-e36dd7e7a17ad87f8a03b9c38b02aa56c8343692.png"/>
</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Nomad has many possible task drivers, but this example uses only a Docker container.</p>
</div>
<div class="admonition important">
<p class="admonition-title">Important</p>
<p><code class="docutils literal notranslate"><span class="pre">ceph-csi</span></code> uses the RBD kernel modules by default, which may not support
all Ceph <a class="reference external" href="../../rados/operations/crush-map/#tunables">CRUSH tunables</a> or <a class="reference external" href="../rbd-config-ref/#image-features">RBD image features</a>.</p>
</div>
<section id="create-a-pool">
<h2>Create a Pool<a class="headerlink" href="#create-a-pool" title="Permalink to this heading"></a></h2>
<p>By default, Ceph block devices use the <code class="docutils literal notranslate"><span class="pre">rbd</span></code> pool. Ensure that your Ceph
cluster is running, then create a pool for Nomad persistent storage:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><style type="text/css">
span.prompt1:before {
  content: "$ ";
}
</style><span class="prompt1">ceph<span class="w"> </span>osd<span class="w"> </span>pool<span class="w"> </span>create<span class="w"> </span>nomad</span>
</pre></div></div><p>See <a class="reference external" href="../../rados/operations/pools#createpool">Create a Pool</a> for details on specifying the number of placement groups
for your pools. See <a class="reference external" href="../../rados/operations/placement-groups">Placement Groups</a> for details on the number of placement
groups you should set for your pools.</p>
<p>A newly created pool must be initialized prior to use. Use the <code class="docutils literal notranslate"><span class="pre">rbd</span></code> tool
to initialize the pool:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">rbd<span class="w"> </span>pool<span class="w"> </span>init<span class="w"> </span>nomad</span>
</pre></div></div></section>
<section id="configure-ceph-csi">
<h2>Configure ceph-csi<a class="headerlink" href="#configure-ceph-csi" title="Permalink to this heading"></a></h2>
<section id="ceph-client-authentication-setup">
<h3>Ceph Client Authentication Setup<a class="headerlink" href="#ceph-client-authentication-setup" title="Permalink to this heading"></a></h3>
<p>Create a new user for Nomad and <cite>ceph-csi</cite>. Execute the following command and
record the generated key:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>ceph<span class="w"> </span>auth<span class="w"> </span>get-or-create<span class="w"> </span>client.nomad<span class="w"> </span>mon<span class="w"> </span><span class="s1">&#39;profile rbd&#39;</span><span class="w"> </span>osd<span class="w"> </span><span class="s1">&#39;profile rbd pool=nomad&#39;</span><span class="w"> </span>mgr<span class="w"> </span><span class="s1">&#39;profile rbd pool=nomad&#39;</span>
<span class="go">[client.nomad]</span>
<span class="go">        key = AQAlh9Rgg2vrDxAARy25T7KHabs6iskSHpAEAQ==</span>
</pre></div>
</div>
</section>
<section id="configure-nomad">
<h3>Configure Nomad<a class="headerlink" href="#configure-nomad" title="Permalink to this heading"></a></h3>
<section id="configuring-nomad-to-allow-containers-to-use-privileged-mode">
<h4>Configuring Nomad to Allow Containers to Use Privileged Mode<a class="headerlink" href="#configuring-nomad-to-allow-containers-to-use-privileged-mode" title="Permalink to this heading"></a></h4>
<p>By default, Nomad doesn’t allow containers to use privileged mode. We must
configure Nomad so that it allows containers to use privileged mode. Edit the
Nomad configuration file by adding the following configuration block to
<cite>/etc/nomad.d/nomad.hcl</cite>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">plugin</span> <span class="s2">&quot;docker&quot;</span> <span class="p">{</span>
  <span class="n">config</span> <span class="p">{</span>
    <span class="n">allow_privileged</span> <span class="o">=</span> <span class="n">true</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
</section>
<section id="loading-the-rbd-module">
<h4>Loading the rbd module<a class="headerlink" href="#loading-the-rbd-module" title="Permalink to this heading"></a></h4>
<p>Nomad must have the <cite>rbd</cite> module loaded. Run the following command to confirm that the <cite>rbd</cite> module is loaded:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>lsmod<span class="w"> </span><span class="p">|</span><span class="w"> </span>grep<span class="w"> </span>rbd
<span class="go">rbd                    94208  2</span>
<span class="go">libceph               364544  1 rbd</span>
</pre></div>
</div>
<p>If the <cite>rbd</cite> module is not loaded, load it:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">sudo<span class="w"> </span>modprobe<span class="w"> </span>rbd</span>
</pre></div></div></section>
<section id="restarting-nomad">
<h4>Restarting Nomad<a class="headerlink" href="#restarting-nomad" title="Permalink to this heading"></a></h4>
<p>Restart Nomad:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">sudo<span class="w"> </span>systemctl<span class="w"> </span>restart<span class="w"> </span>nomad</span>
</pre></div></div></section>
</section>
</section>
<section id="create-ceph-csi-controller-and-plugin-nodes">
<h2>Create ceph-csi controller and plugin nodes<a class="headerlink" href="#create-ceph-csi-controller-and-plugin-nodes" title="Permalink to this heading"></a></h2>
<p>The <a class="reference external" href="https://github.com/ceph/ceph-csi/">ceph-csi</a> plugin requires two components:</p>
<ul class="simple">
<li><p><strong>Controller plugin</strong>: communicates with the provider’s API.</p></li>
<li><p><strong>Node plugin</strong>: executes tasks on the client.</p></li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>We’ll set the ceph-csi’s version in those files. See <a class="reference external" href="https://github.com/ceph/ceph-csi#ceph-csi-container-images-and-release-compatibility">ceph-csi release</a>
for information about ceph-csi’s compatibility with other versions.</p>
</div>
<section id="configure-controller-plugin">
<h3>Configure controller plugin<a class="headerlink" href="#configure-controller-plugin" title="Permalink to this heading"></a></h3>
<p>The controller plugin requires the Ceph monitor addresses of the Ceph
cluster. Collect both (1) the Ceph cluster unique <cite>fsid</cite> and (2) the monitor
addresses:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>ceph<span class="w"> </span>mon<span class="w"> </span>dump
<span class="go">&lt;...&gt;</span>
<span class="go">fsid b9127830-b0cc-4e34-aa47-9d1a2e9949a8</span>
<span class="go">&lt;...&gt;</span>
<span class="go">0: [v2:192.168.1.1:3300/0,v1:192.168.1.1:6789/0] mon.a</span>
<span class="go">1: [v2:192.168.1.2:3300/0,v1:192.168.1.2:6789/0] mon.b</span>
<span class="go">2: [v2:192.168.1.3:3300/0,v1:192.168.1.3:6789/0] mon.c</span>
</pre></div>
</div>
<p>Generate a <code class="docutils literal notranslate"><span class="pre">ceph-csi-plugin-controller.nomad</span></code> file similar to the example
below. Substitute the <cite>fsid</cite> for “clusterID”, and the monitor addresses for
“monitors”:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">job</span> <span class="s2">&quot;ceph-csi-plugin-controller&quot;</span> <span class="p">{</span>
  <span class="n">datacenters</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;dc1&quot;</span><span class="p">]</span>
  <span class="n">group</span> <span class="s2">&quot;controller&quot;</span> <span class="p">{</span>
    <span class="n">network</span> <span class="p">{</span>
      <span class="n">port</span> <span class="s2">&quot;metrics&quot;</span> <span class="p">{}</span>
    <span class="p">}</span>
    <span class="n">task</span> <span class="s2">&quot;ceph-controller&quot;</span> <span class="p">{</span>
      <span class="n">template</span> <span class="p">{</span>
        <span class="n">data</span>        <span class="o">=</span> <span class="o">&lt;&lt;</span><span class="n">EOF</span>
<span class="p">[{</span>
    <span class="s2">&quot;clusterID&quot;</span><span class="p">:</span> <span class="s2">&quot;b9127830-b0cc-4e34-aa47-9d1a2e9949a8&quot;</span><span class="p">,</span>
    <span class="s2">&quot;monitors&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;192.168.1.1&quot;</span><span class="p">,</span>
        <span class="s2">&quot;192.168.1.2&quot;</span><span class="p">,</span>
        <span class="s2">&quot;192.168.1.3&quot;</span>
    <span class="p">]</span>
<span class="p">}]</span>
<span class="n">EOF</span>
        <span class="n">destination</span> <span class="o">=</span> <span class="s2">&quot;local/config.json&quot;</span>
        <span class="n">change_mode</span> <span class="o">=</span> <span class="s2">&quot;restart&quot;</span>
      <span class="p">}</span>
      <span class="n">driver</span> <span class="o">=</span> <span class="s2">&quot;docker&quot;</span>
      <span class="n">config</span> <span class="p">{</span>
        <span class="n">image</span> <span class="o">=</span> <span class="s2">&quot;quay.io/cephcsi/cephcsi:v3.3.1&quot;</span>
        <span class="n">volumes</span> <span class="o">=</span> <span class="p">[</span>
          <span class="s2">&quot;./local/config.json:/etc/ceph-csi-config/config.json&quot;</span>
        <span class="p">]</span>
        <span class="n">mounts</span> <span class="o">=</span> <span class="p">[</span>
          <span class="p">{</span>
            <span class="nb">type</span>     <span class="o">=</span> <span class="s2">&quot;tmpfs&quot;</span>
            <span class="n">target</span>   <span class="o">=</span> <span class="s2">&quot;/tmp/csi/keys&quot;</span>
            <span class="n">readonly</span> <span class="o">=</span> <span class="n">false</span>
            <span class="n">tmpfs_options</span> <span class="o">=</span> <span class="p">{</span>
              <span class="n">size</span> <span class="o">=</span> <span class="mi">1000000</span> <span class="c1"># size in bytes</span>
            <span class="p">}</span>
          <span class="p">}</span>
        <span class="p">]</span>
        <span class="n">args</span> <span class="o">=</span> <span class="p">[</span>
          <span class="s2">&quot;--type=rbd&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--controllerserver=true&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--drivername=rbd.csi.ceph.com&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--endpoint=unix://csi/csi.sock&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--nodeid=$</span><span class="si">{node.unique.name}</span><span class="s2">&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--instanceid=$</span><span class="si">{node.unique.name}</span><span class="s2">-controller&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--pidlimit=-1&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--logtostderr=true&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--v=5&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--metricsport=$$</span><span class="si">{NOMAD_PORT_metrics}</span><span class="s2">&quot;</span>
        <span class="p">]</span>
      <span class="p">}</span>
      <span class="n">resources</span> <span class="p">{</span>
        <span class="n">cpu</span>    <span class="o">=</span> <span class="mi">500</span>
        <span class="n">memory</span> <span class="o">=</span> <span class="mi">256</span>
      <span class="p">}</span>
      <span class="n">service</span> <span class="p">{</span>
        <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;ceph-csi-controller&quot;</span>
        <span class="n">port</span> <span class="o">=</span> <span class="s2">&quot;metrics&quot;</span>
        <span class="n">tags</span> <span class="o">=</span> <span class="p">[</span> <span class="s2">&quot;prometheus&quot;</span> <span class="p">]</span>
      <span class="p">}</span>
      <span class="n">csi_plugin</span> <span class="p">{</span>
        <span class="nb">id</span>        <span class="o">=</span> <span class="s2">&quot;ceph-csi&quot;</span>
        <span class="nb">type</span>      <span class="o">=</span> <span class="s2">&quot;controller&quot;</span>
        <span class="n">mount_dir</span> <span class="o">=</span> <span class="s2">&quot;/csi&quot;</span>
      <span class="p">}</span>
    <span class="p">}</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
</section>
<section id="configure-plugin-node">
<h3>Configure plugin node<a class="headerlink" href="#configure-plugin-node" title="Permalink to this heading"></a></h3>
<p>Generate a <code class="docutils literal notranslate"><span class="pre">ceph-csi-plugin-nodes.nomad</span></code> file similar to the example below.
Substitute the <cite>fsid</cite> for “clusterID” and the monitor addresses for
“monitors”:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">job</span> <span class="s2">&quot;ceph-csi-plugin-nodes&quot;</span> <span class="p">{</span>
  <span class="n">datacenters</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;dc1&quot;</span><span class="p">]</span>
  <span class="nb">type</span>        <span class="o">=</span> <span class="s2">&quot;system&quot;</span>
  <span class="n">group</span> <span class="s2">&quot;nodes&quot;</span> <span class="p">{</span>
    <span class="n">network</span> <span class="p">{</span>
      <span class="n">port</span> <span class="s2">&quot;metrics&quot;</span> <span class="p">{}</span>
    <span class="p">}</span>
    <span class="n">task</span> <span class="s2">&quot;ceph-node&quot;</span> <span class="p">{</span>
      <span class="n">driver</span> <span class="o">=</span> <span class="s2">&quot;docker&quot;</span>
      <span class="n">template</span> <span class="p">{</span>
        <span class="n">data</span>        <span class="o">=</span> <span class="o">&lt;&lt;</span><span class="n">EOF</span>
<span class="p">[{</span>
    <span class="s2">&quot;clusterID&quot;</span><span class="p">:</span> <span class="s2">&quot;b9127830-b0cc-4e34-aa47-9d1a2e9949a8&quot;</span><span class="p">,</span>
    <span class="s2">&quot;monitors&quot;</span><span class="p">:</span> <span class="p">[</span>
        <span class="s2">&quot;192.168.1.1&quot;</span><span class="p">,</span>
        <span class="s2">&quot;192.168.1.2&quot;</span><span class="p">,</span>
        <span class="s2">&quot;192.168.1.3&quot;</span>
    <span class="p">]</span>
<span class="p">}]</span>
<span class="n">EOF</span>
        <span class="n">destination</span> <span class="o">=</span> <span class="s2">&quot;local/config.json&quot;</span>
        <span class="n">change_mode</span> <span class="o">=</span> <span class="s2">&quot;restart&quot;</span>
      <span class="p">}</span>
      <span class="n">config</span> <span class="p">{</span>
        <span class="n">image</span> <span class="o">=</span> <span class="s2">&quot;quay.io/cephcsi/cephcsi:v3.3.1&quot;</span>
        <span class="n">volumes</span> <span class="o">=</span> <span class="p">[</span>
          <span class="s2">&quot;./local/config.json:/etc/ceph-csi-config/config.json&quot;</span>
        <span class="p">]</span>
        <span class="n">mounts</span> <span class="o">=</span> <span class="p">[</span>
          <span class="p">{</span>
            <span class="nb">type</span>     <span class="o">=</span> <span class="s2">&quot;tmpfs&quot;</span>
            <span class="n">target</span>   <span class="o">=</span> <span class="s2">&quot;/tmp/csi/keys&quot;</span>
            <span class="n">readonly</span> <span class="o">=</span> <span class="n">false</span>
            <span class="n">tmpfs_options</span> <span class="o">=</span> <span class="p">{</span>
              <span class="n">size</span> <span class="o">=</span> <span class="mi">1000000</span> <span class="c1"># size in bytes</span>
            <span class="p">}</span>
          <span class="p">}</span>
        <span class="p">]</span>
        <span class="n">args</span> <span class="o">=</span> <span class="p">[</span>
          <span class="s2">&quot;--type=rbd&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--drivername=rbd.csi.ceph.com&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--nodeserver=true&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--endpoint=unix://csi/csi.sock&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--nodeid=$</span><span class="si">{node.unique.name}</span><span class="s2">&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--instanceid=$</span><span class="si">{node.unique.name}</span><span class="s2">-nodes&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--pidlimit=-1&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--logtostderr=true&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--v=5&quot;</span><span class="p">,</span>
          <span class="s2">&quot;--metricsport=$$</span><span class="si">{NOMAD_PORT_metrics}</span><span class="s2">&quot;</span>
        <span class="p">]</span>
        <span class="n">privileged</span> <span class="o">=</span> <span class="n">true</span>
      <span class="p">}</span>
      <span class="n">resources</span> <span class="p">{</span>
        <span class="n">cpu</span>    <span class="o">=</span> <span class="mi">500</span>
        <span class="n">memory</span> <span class="o">=</span> <span class="mi">256</span>
      <span class="p">}</span>
      <span class="n">service</span> <span class="p">{</span>
        <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;ceph-csi-nodes&quot;</span>
        <span class="n">port</span> <span class="o">=</span> <span class="s2">&quot;metrics&quot;</span>
        <span class="n">tags</span> <span class="o">=</span> <span class="p">[</span> <span class="s2">&quot;prometheus&quot;</span> <span class="p">]</span>
      <span class="p">}</span>
      <span class="n">csi_plugin</span> <span class="p">{</span>
        <span class="nb">id</span>        <span class="o">=</span> <span class="s2">&quot;ceph-csi&quot;</span>
        <span class="nb">type</span>      <span class="o">=</span> <span class="s2">&quot;node&quot;</span>
        <span class="n">mount_dir</span> <span class="o">=</span> <span class="s2">&quot;/csi&quot;</span>
      <span class="p">}</span>
    <span class="p">}</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
</section>
<section id="start-plugin-controller-and-node">
<h3>Start plugin controller and node<a class="headerlink" href="#start-plugin-controller-and-node" title="Permalink to this heading"></a></h3>
<p>To start the plugin controller and the Nomad node, run the following commands:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">nomad<span class="w"> </span>job<span class="w"> </span>run<span class="w"> </span>ceph-csi-plugin-controller.nomad</span>
<span class="prompt1">nomad<span class="w"> </span>job<span class="w"> </span>run<span class="w"> </span>ceph-csi-plugin-nodes.nomad</span>
</pre></div></div><p>The <a class="reference external" href="https://github.com/ceph/ceph-csi/">ceph-csi</a> image will be downloaded.</p>
<p>Check the plugin status after a few minutes:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>nomad<span class="w"> </span>plugin<span class="w"> </span>status<span class="w"> </span>ceph-csi
<span class="go">ID                   = ceph-csi</span>
<span class="go">Provider             = rbd.csi.ceph.com</span>
<span class="go">Version              = 3.3.1</span>
<span class="go">Controllers Healthy  = 1</span>
<span class="go">Controllers Expected = 1</span>
<span class="go">Nodes Healthy        = 1</span>
<span class="go">Nodes Expected       = 1</span>

<span class="go">Allocations</span>
<span class="go">ID        Node ID   Task Group  Version  Desired  Status   Created    Modified</span>
<span class="go">23b4db0c  a61ef171  nodes       4        run      running  3h26m ago  3h25m ago</span>
<span class="go">fee74115  a61ef171  controller  6        run      running  3h26m ago  3h25m ago</span>
</pre></div>
</div>
</section>
</section>
<section id="using-ceph-block-devices">
<h2>Using Ceph Block Devices<a class="headerlink" href="#using-ceph-block-devices" title="Permalink to this heading"></a></h2>
<section id="create-rbd-image">
<h3>Create rbd image<a class="headerlink" href="#create-rbd-image" title="Permalink to this heading"></a></h3>
<p><code class="docutils literal notranslate"><span class="pre">ceph-csi</span></code> requires the cephx credentials for communicating with the Ceph
cluster. Generate a <code class="docutils literal notranslate"><span class="pre">ceph-volume.hcl</span></code> file similar to the example below,
using the newly created nomad user id and cephx key:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nb">id</span> <span class="o">=</span> <span class="s2">&quot;ceph-mysql&quot;</span>
<span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;ceph-mysql&quot;</span>
<span class="nb">type</span> <span class="o">=</span> <span class="s2">&quot;csi&quot;</span>
<span class="n">plugin_id</span> <span class="o">=</span> <span class="s2">&quot;ceph-csi&quot;</span>
<span class="n">capacity_max</span> <span class="o">=</span> <span class="s2">&quot;200G&quot;</span>
<span class="n">capacity_min</span> <span class="o">=</span> <span class="s2">&quot;100G&quot;</span>

<span class="n">capability</span> <span class="p">{</span>
  <span class="n">access_mode</span>     <span class="o">=</span> <span class="s2">&quot;single-node-writer&quot;</span>
  <span class="n">attachment_mode</span> <span class="o">=</span> <span class="s2">&quot;file-system&quot;</span>
<span class="p">}</span>

<span class="n">secrets</span> <span class="p">{</span>
  <span class="n">userID</span>  <span class="o">=</span> <span class="s2">&quot;admin&quot;</span>
  <span class="n">userKey</span> <span class="o">=</span> <span class="s2">&quot;AQAlh9Rgg2vrDxAARy25T7KHabs6iskSHpAEAQ==&quot;</span>
<span class="p">}</span>

<span class="n">parameters</span> <span class="p">{</span>
  <span class="n">clusterID</span> <span class="o">=</span> <span class="s2">&quot;b9127830-b0cc-4e34-aa47-9d1a2e9949a8&quot;</span>
  <span class="n">pool</span> <span class="o">=</span> <span class="s2">&quot;nomad&quot;</span>
  <span class="n">imageFeatures</span> <span class="o">=</span> <span class="s2">&quot;layering&quot;</span>
<span class="p">}</span>
</pre></div>
</div>
<p>After the <code class="docutils literal notranslate"><span class="pre">ceph-volume.hcl</span></code> file has been generated, create the volume:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">nomad<span class="w"> </span>volume<span class="w"> </span>create<span class="w"> </span>ceph-volume.hcl</span>
</pre></div></div></section>
<section id="use-rbd-image-with-a-container">
<h3>Use rbd image with a container<a class="headerlink" href="#use-rbd-image-with-a-container" title="Permalink to this heading"></a></h3>
<p>As an exercise in using an rbd image with a container, modify the Hashicorp
<a class="reference external" href="https://learn.hashicorp.com/tutorials/nomad/stateful-workloads-csi-volumes?in=nomad/stateful-workloads#create-the-job-file">nomad stateful</a> example.</p>
<p>Generate a <code class="docutils literal notranslate"><span class="pre">mysql.nomad</span></code> file similar to the example below:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">job</span> <span class="s2">&quot;mysql-server&quot;</span> <span class="p">{</span>
  <span class="n">datacenters</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;dc1&quot;</span><span class="p">]</span>
  <span class="nb">type</span>        <span class="o">=</span> <span class="s2">&quot;service&quot;</span>
  <span class="n">group</span> <span class="s2">&quot;mysql-server&quot;</span> <span class="p">{</span>
    <span class="n">count</span> <span class="o">=</span> <span class="mi">1</span>
    <span class="n">volume</span> <span class="s2">&quot;ceph-mysql&quot;</span> <span class="p">{</span>
      <span class="nb">type</span>      <span class="o">=</span> <span class="s2">&quot;csi&quot;</span>
      <span class="n">attachment_mode</span> <span class="o">=</span> <span class="s2">&quot;file-system&quot;</span>
      <span class="n">access_mode</span>     <span class="o">=</span> <span class="s2">&quot;single-node-writer&quot;</span>
      <span class="n">read_only</span> <span class="o">=</span> <span class="n">false</span>
      <span class="n">source</span>    <span class="o">=</span> <span class="s2">&quot;ceph-mysql&quot;</span>
    <span class="p">}</span>
    <span class="n">network</span> <span class="p">{</span>
      <span class="n">port</span> <span class="s2">&quot;db&quot;</span> <span class="p">{</span>
        <span class="n">static</span> <span class="o">=</span> <span class="mi">3306</span>
      <span class="p">}</span>
    <span class="p">}</span>
    <span class="n">restart</span> <span class="p">{</span>
      <span class="n">attempts</span> <span class="o">=</span> <span class="mi">10</span>
      <span class="n">interval</span> <span class="o">=</span> <span class="s2">&quot;5m&quot;</span>
      <span class="n">delay</span>    <span class="o">=</span> <span class="s2">&quot;25s&quot;</span>
      <span class="n">mode</span>     <span class="o">=</span> <span class="s2">&quot;delay&quot;</span>
    <span class="p">}</span>
    <span class="n">task</span> <span class="s2">&quot;mysql-server&quot;</span> <span class="p">{</span>
      <span class="n">driver</span> <span class="o">=</span> <span class="s2">&quot;docker&quot;</span>
      <span class="n">volume_mount</span> <span class="p">{</span>
        <span class="n">volume</span>      <span class="o">=</span> <span class="s2">&quot;ceph-mysql&quot;</span>
        <span class="n">destination</span> <span class="o">=</span> <span class="s2">&quot;/srv&quot;</span>
        <span class="n">read_only</span>   <span class="o">=</span> <span class="n">false</span>
      <span class="p">}</span>
      <span class="n">env</span> <span class="p">{</span>
        <span class="n">MYSQL_ROOT_PASSWORD</span> <span class="o">=</span> <span class="s2">&quot;password&quot;</span>
      <span class="p">}</span>
      <span class="n">config</span> <span class="p">{</span>
        <span class="n">image</span> <span class="o">=</span> <span class="s2">&quot;hashicorp/mysql-portworx-demo:latest&quot;</span>
        <span class="n">args</span>  <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;--datadir&quot;</span><span class="p">,</span> <span class="s2">&quot;/srv/mysql&quot;</span><span class="p">]</span>
        <span class="n">ports</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;db&quot;</span><span class="p">]</span>
      <span class="p">}</span>
      <span class="n">resources</span> <span class="p">{</span>
        <span class="n">cpu</span>    <span class="o">=</span> <span class="mi">500</span>
        <span class="n">memory</span> <span class="o">=</span> <span class="mi">1024</span>
      <span class="p">}</span>
      <span class="n">service</span> <span class="p">{</span>
        <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;mysql-server&quot;</span>
        <span class="n">port</span> <span class="o">=</span> <span class="s2">&quot;db&quot;</span>
        <span class="n">check</span> <span class="p">{</span>
          <span class="nb">type</span>     <span class="o">=</span> <span class="s2">&quot;tcp&quot;</span>
          <span class="n">interval</span> <span class="o">=</span> <span class="s2">&quot;10s&quot;</span>
          <span class="n">timeout</span>  <span class="o">=</span> <span class="s2">&quot;2s&quot;</span>
        <span class="p">}</span>
      <span class="p">}</span>
    <span class="p">}</span>
  <span class="p">}</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Start the job:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">nomad<span class="w"> </span>job<span class="w"> </span>run<span class="w"> </span>mysql.nomad</span>
</pre></div></div><p>Check the status of the job:</p>
<div class="highlight-console notranslate"><div class="highlight"><pre><span></span><span class="gp">$ </span>nomad<span class="w"> </span>job<span class="w"> </span>status<span class="w"> </span>mysql-server
<span class="go">...</span>
<span class="go">Status        = running</span>
<span class="go">...</span>
<span class="go">Allocations</span>
<span class="go">ID        Node ID   Task Group    Version  Desired  Status   Created  Modified</span>
<span class="go">38070da7  9ad01c63  mysql-server  0        run      running  6s ago   3s ago</span>
</pre></div>
</div>
<p>To check that data are persistent, modify the database, purge the job, then
create it using the same file. The same RBD image will be used (re-used,
really).</p>
</section>
</section>
</section>



<div id="support-the-ceph-foundation" class="admonition note">
  <p class="first admonition-title">Brought to you by the Ceph Foundation</p>
  <p class="last">The Ceph Documentation is a community resource funded and hosted by the non-profit <a href="https://ceph.io/en/foundation/">Ceph Foundation</a>. If you would like to support this and our other efforts, please consider <a href="https://ceph.io/en/foundation/join/">joining now</a>.</p>
</div>


           </div>
           
          </div>
          <footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
        <a href="../rbd-kubernetes/" class="btn btn-neutral float-left" title="Block Devices and Kubernetes" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
        <a href="../rbd-openstack/" class="btn btn-neutral float-right" title="块设备与 OpenStack" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
    </div>

  <hr/>

  <div role="contentinfo">
    <p>&#169; Copyright 2016, Ceph authors and contributors. Licensed under Creative Commons Attribution Share Alike 3.0 (CC-BY-SA-3.0).</p>
  </div>

   

</footer>
        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>